{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第五步：调整正则化参数 ：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read data\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath + 'RentListingInquries_FE_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "train = train.drop(['interest_level'], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58138, std: 0.00341, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58103, std: 0.00350, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58131, std: 0.00333, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58121, std: 0.00416, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58116, std: 0.00376, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58089, std: 0.00358, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 2},\n",
       " -0.58088992143281715)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(learning_rate=0.1, \n",
    "                       n_estimators=313, # 上一轮得出\n",
    "                       max_depth=5,                       \n",
    "                       min_child_weight=5, \n",
    "                       gamma=0, \n",
    "                       subsample=0.8, \n",
    "                       colsample_bytree=0.9, \n",
    "                       colsample_bylevel=0.7, \n",
    "                       objective= 'multi:softprob',\n",
    "                       seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 314.43525343,  298.13167024,  293.40243721,  322.36138206,\n",
       "         302.36139178,  255.55085058]),\n",
       " 'mean_score_time': array([ 1.37074118,  1.15232882,  1.15456338,  1.25849352,  1.07053542,\n",
       "         1.10190067]),\n",
       " 'mean_test_score': array([-0.58138052, -0.58102585, -0.58130633, -0.58121397, -0.58116153,\n",
       "        -0.58088992]),\n",
       " 'mean_train_score': array([-0.4855644 , -0.48649149, -0.48824055, -0.48648381, -0.48770842,\n",
       "        -0.48938765]),\n",
       " 'param_reg_alpha': masked_array(data = [1.5 1.5 1.5 2 2 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_reg_lambda': masked_array(data = [0.5 1 2 0.5 1 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([6, 2, 5, 4, 3, 1], dtype=int32),\n",
       " 'split0_test_score': array([-0.57553257, -0.57501989, -0.57557159, -0.57442887, -0.57505448,\n",
       "        -0.57498303]),\n",
       " 'split0_train_score': array([-0.48705457, -0.48817446, -0.48988929, -0.48778397, -0.48933696,\n",
       "        -0.49066186]),\n",
       " 'split1_test_score': array([-0.58012071, -0.57967215, -0.58057637, -0.57903777, -0.5789139 ,\n",
       "        -0.57928646]),\n",
       " 'split1_train_score': array([-0.48560626, -0.48691483, -0.48818772, -0.48602571, -0.48778909,\n",
       "        -0.48953164]),\n",
       " 'split2_test_score': array([-0.58188402, -0.58162625, -0.58130158, -0.58180481, -0.58232362,\n",
       "        -0.58123467]),\n",
       " 'split2_train_score': array([-0.4844579 , -0.48533588, -0.48820295, -0.48593258, -0.48720703,\n",
       "        -0.48806709]),\n",
       " 'split3_test_score': array([-0.58422641, -0.58404725, -0.58380888, -0.58489572, -0.58395682,\n",
       "        -0.58385333]),\n",
       " 'split3_train_score': array([-0.48572545, -0.48650422, -0.48833418, -0.48646649, -0.48731745,\n",
       "        -0.48934136]),\n",
       " 'split4_test_score': array([-0.58514002, -0.58476486, -0.58527441, -0.58590412, -0.58556014,\n",
       "        -0.5850934 ]),\n",
       " 'split4_train_score': array([-0.4849778 , -0.48552808, -0.48658862, -0.48621031, -0.48689158,\n",
       "        -0.48933627]),\n",
       " 'std_fit_time': array([  3.66158883,   8.15233532,   4.18335445,  12.7067447 ,\n",
       "          2.16731698,  63.62486362]),\n",
       " 'std_score_time': array([ 0.07274457,  0.16326207,  0.09153915,  0.11388616,  0.0847337 ,\n",
       "         0.32158151]),\n",
       " 'std_test_score': array([ 0.00341384,  0.00350352,  0.00332948,  0.00416351,  0.00376452,\n",
       "         0.00357857]),\n",
       " 'std_train_score': array([ 0.0008737 ,  0.00102717,  0.00104501,  0.00067516,  0.00086373,\n",
       "         0.00082377])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.580890 using {'reg_alpha': 2, 'reg_lambda': 2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4VNXWwOHfSoFA6BBqKBECoQcILRSplyYX9YKiiIoK\noiKIBcWKfHaQCyiKYL0WwC5Kld6kBAidQAgttISggQDp+/vjDBgwkAAzOTPJep9nHmbOnLIGSNbs\nvdfZW4wxKKWUUs7mZXcASiml8idNMEoppVxCE4xSSimX0ASjlFLKJTTBKKWUcglNMEoppVxCE4xS\nSimX0ASjlFLKJVyaYESku4hEiUi0iDyXzfsdRCRRRCIdj5ezvDdSRHaIyHYRmSEifo7t/RzbM0Uk\nLJtzVhORJBF52pWfTSml1NX5uOrEIuINTAG6ArHABhGZbYzZedmuK40xt1x2bBVgOFDPGHNeRL4F\n+gOfA9uB24GPrnDpCcC83MRYrlw5U6NGjdx9IKWUUgBs3LjxpDEmIKf9XJZggBZAtDEmBkBEZgJ9\ngMsTzJX4AEVEJA0oChwFMMbscpzvHweIyK3AfuBsbi5Qo0YNIiIichmOUkopABE5mJv9XNlFVgU4\nnOV1rGPb5cJFZKuIzBOR+gDGmCPAeOAQcAxINMYsvNrFRKQY8Czwag77DRGRCBGJiI+Pz/2nUUop\ndU3sHuTfBFQzxjQC3gN+BhCR0litnSCgMuAvIvfkcK4xwH+NMUlX28kYM80YE2aMCQsIyLGFp5RS\n6jq5MsEcAapmeR3o2HaRMeb0hYRgjJkL+IpIOaALsN8YE2+MSQN+BMJzuF5L4B0ROQA8ATwvIsOc\n8kmUUkpdM1eOwWwAgkUkCCux9AfuzrqDiFQEThhjjIi0wEp4CVhdY61EpChwHugMXHWwxBjTLst5\nxwBJxpj3nfdxlFJKXQuXJRhjTLqjBbEA8AY+NcbsEJGhjvenAn2BR0QkHSuR9DfWAjXrROR7rC60\ndGAzMA1ARG7D6k4LAOaISKQxppurPodSSqnrIwV5wbGwsDCjVWRKKXVtRGSjMeYf9yFezu5BfqWU\nUvmUJpjrkJyWwZSl0SSnZdgdilJKuS1NMNdhy+G/GLcgijfn7rI7FKWUcluaYK5Dy5vK8mDbIL74\n4yBLd8fZHY5SSrklTTDX6ZludQipWJxnvt/CyaQUu8NRSim3ownmOvn5ejP5riacTk5n1PdbKcjV\neEoplR1NMDegdoXiPN8jhCW74/hqba7mflNKqQJDE8wNui+8BjfXDuC1ObvYe+KM3eEopZTb0ARz\ng0SEcf0aUaywD8NnRpKSrqXLSikFmmCconxxP97p24hdx07z7sI9doejlFJuQROMk3SuW4F7WlVj\n2ooYVu09aXc4SillO00wTvRCz3rUDPDnqe8i+fNsqt3hKKWUrTTBOFGRQt5M6t+EU2dTGf3jNi1d\nVkoVaJpgnKxBlZI8/a86zN9xnO8iYu0ORymlbKMJxgUGt7uJ8JplGfPrDvafPGt3OEopZQtNMC7g\n5SW8e0djfL29eGLmZtIyMu0OSSml8pwmGBepVLIIb93ekC2xiUxatNfucJRSKs9pgnGhHg0r0a9Z\nIB8si2b9/lN2h6OUUnlKE4yLvfLv+lQtU5SRsyJJPJ9mdzhKKZVnNMG4WLHCPky8M5Tjp5N5+Zft\ndoejlFJ5RhNMHmhSrTRPdA7ml8ij/Lz5iN3hKKVUntAEk0ce7ViLsOqleenn7Rw+dc7ucJRSyuU0\nweQRby/hv3eGAjByViTpWrqslMrnNMHkoaplivJ/tzYg4uCffLhsn93hKKWUS2mCyWO3NqlCn9DK\nTFy8l82H/rQ7HKWUchlNMDYY26cBFUv48cSsSM6mpNsdjlJKuYQmGBuULOLLf+8M5fCpc7z66w67\nw1FKKZfQBGOTFkFleLRDLb6NiGXutmN2h6OUUk6nCcZGI7oE0ziwJKN/3MaxxPN2h6OUUk6lCcZG\nvt5eTOzfhLSMTJ76dguZmbpAmVIq/9AEY7Ogcv680rsea/Yl8PGqGLvDUUopp3FpghGR7iISJSLR\nIvJcNu93EJFEEYl0PF7O8t5IEdkhIttFZIaI+Dm293NszxSRsCz7t8hyni0icpsrP5sz3RFWle71\nKzJuQRTbjyTaHY5SKh9Lz8jkq7UHmb/d9WO/LkswIuINTAF6APWAu0SkXja7rjTGhDoeYx3HVgGG\nA2HGmAaAN9Dfsf924HZgxWXn2e7YPxToDnwkIj7O/lyuICK8eXtDyvgXYsTMzZxPzbA7JKVUPmOM\nYcnuE3SftJIXf97OvO3HXX5NV7ZgWgDRxpgYY0wqMBPocw3H+wBFHEmiKHAUwBizyxgTdfnOxphz\nxpgLN5X4AR41oFHavxDv9gtlX/xZ3pi7y+5wlFL5yI6jidzzyToe+DyCjEzDRwObMdExdZUrufIb\nfhXgcJbXsUDLbPYLF5GtwBHgaWPMDmPMEREZDxwCzgMLjTELc7qgiLQEPgWqAwOzJByP0Da4HIPb\nBTF95X461Amgc90KdoeklPJgxxOTGb8wih82xVKqiC9jetdjQKvq+HrnzfC73YP8m4BqxphGwHvA\nzwAiUhqrtRMEVAb8ReSenE5mjFlnjKkPNAdGXxi3yUpEhohIhIhExMfHO/GjOMfT3epQt1IJRn2/\nlbgzyXaHo5TyQGdT0pmwMIoO45cyO/IoQ9rdxLJnOnJ/m6A8Sy7g2gRzBKia5XWgY9tFxpjTxpgk\nx/O5gK+IlAO6APuNMfHGmDTgRyA8txc2xuwCkoAG2bw3zRgTZowJCwgIuNbP5HKFfbyZ3D+UpJR0\nnvluK8Z4VE+fUspGGZmGGesPcfO4ZUxeEk3XehVZ/NTNjO5Zl5JFfPM8HlcmmA1AsIgEiUghrEH6\n2Vl3EJGKIiKO5y0c8SRgdY21EpGijvc7A1cdmHBcx8fxvDoQAhxw7kfKG8EVivNCr7os3xPP//44\naHc4SikPsHxPPD0nrWT0j9uoUbYoPz0aznt3NaFqmaK2xeSyMRhjTLqIDAMWYFWBfWqM2SEiQx3v\nTwX6Ao+ISDrWWEt/Y31lXyci32N1oaUDm4FpAI7y4/eAAGCOiEQaY7oBbYHnRCQNyAQeNcacdNXn\nc7WBraqzdHccr8/dReuaZaldobjdISml3NDu46d5fc4uVu49SfWyRflwQFO6N6iI47u7raQgd8GE\nhYWZiIgIu8O4ovgzKfSYtIJyxQrz82Nt8PP1tjskpZSbiDudzLsL9/DdxsMU9/NleOdgBraqTiEf\n14+xiMhGY0xYTvt5xH0iBVVA8cKM69uYQZ9vYNyCKF66JbvbiJRSBcm51HSmrYhh2ooY0jIyGdQm\niMc71aJU0UJ2h/YPmmDcXMeQ8tzbujqfrLJKl9sFu19hglLK9TIyDT9sjGX8wijizqTQs2FFnu0e\nQvWy/naHdkWaYDzA8z3rsmZfAk99u4X5T7SnjL/7fVNRSrnOqr0neW3OTnYfP0OTaqX48J6mNKte\nxu6wcmT3fTAqF/x8vZnUP5S/zqXx7A9auqxUQbHnxBnu/2w993yyjrOp6bx/dxN+fCTcI5ILaAvG\nY9SvXJJR3evw2pxdzNxwmLtaVLM7JKWUi8SfSWHC73uYteEQ/oV9eKFnXe4Nr05hH88q9NEE40Ee\naBPEsqh4xv66k5ZBZbgpoJjdISmlnOh8agYfr4xh6vJ9pKRncl94DYZ3Cqa0h3aLaxeZB/HyEsb3\na0xhXy9GzIwkNT3T7pCUUk6Q6RjA7/TuMt79fQ9tg8vx+5M380rv+h6bXEATjMepWNKPt25vxLYj\niUxctMfucJRSN2jNvpP0fn8VT323hfLFC/Ptw635aGAYQeXctzost7SLzAN1b1CR/s2r8uHyfbSv\nHUCrm8raHZJS6hpFxyXx5txdLN4dR5VSRZjUP5TejSrj5WX/HfjOognGQ710Sz3WxiTw5KxI5j3R\n3paJ7JRS1+5kUgqTFu3lm/WHKOrrzbPdQxjUpka+nKlDu8g8lH9hHyb1b0LcmRRe+Gmbli4r5eaS\n0zL4YFk0HcYt45v1hxjQshrLnunAIx1q5svkAtqC8WiNq5ZiZNfajFsQRaeQ8tzeNNDukJRSl8nM\nNMzecpRxC6I48td5utStwOieIdQsAFWgmmA83NCba7I8Kp6Xf9lB8xplbJ2aWyl1qXUxCbw+dxdb\nYxNpUKUE4/s1pnXNgjNmql1kHs7bS5hwZ2NE4IlZkaRnaOmyUnaLiU9iyP8iuHPaWuumyTsaM/ux\ntgUquYC2YPKFwNJFee3WBoyYGcmUpfsY0SXY7pCUKpBOnU1l8uK9fLX2IIV9vHimWx0ebBuUb8dY\ncqIJJp/oE1qFZVHxTF6yl7bB5WhWvbTdISlVYCSnZfDFmgO8vzSasynp3NWiGk90qU1A8cJ2h2Yr\nTTD5yKt96rPhwClGzopk7oh2FCus/7xKuZIxhl+3HuOd+buJ/fM8nULKM7pHCMG6Ai2gYzD5Sgk/\nXybeGUrsn+cYM3uH3eEola9FHDjFbR+sYfiMzRT38+Xrh1ry6f3NNblkoV9x85mwGmUY1rEWk5dE\n06FOALc0qmx3SErlKwdOnuXt+buZt/04FUoUZlzfRtzeNBDvfHQHvrNogsmHHu8czIq9J3n+x200\nrVaayqWK2B2SUh7vr3OpTF4czZdrD+Dr7cWTXWvzULsgihbSX6NXol1k+ZCvtxeT+oeSkWl48ttI\nMjL1Ln+lrldKujWFfvt3lvL5mv30bRbIsqc7MLxzsCaXHOjfTj5Vvaw/r/y7PqO+38r0lTEMvbmm\n3SEp5VGMMczddpy35+/m0Klz3Fw7gNE9QwipWMLu0DyGJph8rF+zQJZFxfHuwija1CxHw8CSdoek\nlEfYePBPXp+zk02H/iKkYnH+90AL2tcOsDssj6MJJh8TEd64rSGbDv7FiFmb+e3xttqkV+oqDp86\nx1vzdzNn6zECihfm7f80pG+zqjqAf510DCafK1W0EBPubMz+k2d5bc4uu8NRyi0lnkvj9Tk76fzu\ncpbsimNE52CWPd2BO5tX0+RyA/TrbAEQXrMcQ9rdxEcrYuhYpzxd61WwOySl3EJqeiZfrT3I5CV7\nSTyfRr9mgTzZtQ4VS/rZHVq+oAmmgHjyX7VZFX2SZ3/YSuPAdpQvoT9AquAyxrBgxwnemreLAwnn\naFurHM/3rEu9yjqA70zaRVZAFPbxZlL/UM6lpvPUd1vI1NJlVUBFHv6LOz9ay9CvNuLr7cVng5rz\n5YMtNLm4gCaY65F6Drb/CH8eAA9aSbJW+eK82KseK/ee5PM1B+wOR6k8FfvnOYbP2MytU1YTczKJ\n129rwLwR7ehYpzwiOs7iCtpFdj2ORcL3g6znRctC5aZQpSlUaWY9L+a+5YwDWlZjWVQcb83fTXit\nslrTr/K908lpTFkazWerDyDAsI61GNqhpk4GmwekIK/lHhYWZiIiIq79wPRUiNsBRzZZj6ObIH43\nGMdiXyWrWgnnQuKpFAp+7vOL/GRSCt0nrqSsfyF+GdamwK5VofK3tIxMvll3iEmL9/LnuVRua1KF\nZ7rVoVJJnTrpRonIRmNMWI77uTLBiEh3YBLgDXxsjHnrsvc7AL8A+x2bfjTGjHW8NxJ4CDDANmCQ\nMSZZRPoBY4C6QAtjTIRj/67AW0AhIBV4xhiz5GrxXXeCyU5KEhzfCkc2/p10/jxw4ZNCudqXtnIq\nNgAf+9aKWBYVx/2fbWBQmxq80ru+bXEo5WzGGBbtiuPNebuIiT9L65vK8kKvujSoojcaO0tuE4zL\n2ogi4g1MAboCscAGEZltjNl52a4rjTG3XHZsFWA4UM8Yc15EvgX6A58D24HbgY8uO89JoLcx5qiI\nNAAWAFWc/LGurHAxqB5uPS44mwBHN1vJ5shGiF4MW2ZY73n5QoX6VsK50NoJqANeedOa6FCnPPeH\n1+Cz1QfoUKc8N+tdyiof2BabyOtzd7I25hQ3Bfjz8b1hdK6rYyx2cWUnZAsg2hgTAyAiM4E+wOUJ\n5kp8gCIikgYUBY4CGGN2Oc53yc7GmM1ZXu5wHFvYGJNyIx/ihviXheAu1gOsgoDTRy5t5Wz9FiI+\nsd739YfKoVC5yd+Jp1R1cNEPx3M9Qliz7yRPf7eF+SPaUbZYwV59T3muo3+dZ9yCKH7afISy/oX4\nv1sb0L95VXy9tY7JTq5MMFWAw1lexwIts9kvXES2AkeAp40xO4wxR0RkPHAIOA8sNMYsvIZr/wfY\nZGtyyY4IlAy0HvX6WNsyMyEh+u9WzpFNsH46ZLxvvZ+1iODCn8XKOyUcP19vJvVvQp/3V/PsD9uY\nfm8z/aanPMqZ5DSmLt/Hxyv3Y4BHOtTkkQ41KeHna3doCvuryDYB1YwxSSLSE/gZCBaR0litnSDg\nL+A7EbnHGPNVTicUkfrA28C/rvD+EGAIQLVq1ZzzKW6ElxcE1LYejftb27IWERx1FBLsW3xpEUHW\nVs4NFBHUrVSCZ3uE8H+/7eSb9YcY0LK6kz6YUq6TnpHJzA2HmbhoDyeTUrk1tDJPd6tDYOmidoem\nsnBlgjkCVM3yOtCx7SJjzOksz+eKyAciUg7oCOw3xsQDiMiPQDhw1QQjIoHAT8C9xph92e1jjJkG\nTANrkP9aP1Se8ClkJZDKTYAHrW3ZFRHsmu04IEsRQWVHIcE1FBEMCq/Bsqg4/u+3nbQMKkut8sVc\n8rGUulHGGJZGxfHG3N1ExyXRIqgMn95fl0aBpewOTWXDlQlmA1ZrJAgrsfQH7s66g4hUBE4YY4yI\ntMC68TMBq2uslYgUxeoi6wxctdxLREoBc4DnjDGrnf1hbJdjEcGm6y4i8PIS3u3XmG4TV/DErM38\n+EgbCvlo37VyLzuOJvLG3F2sjk4gqJw/Hw1sxr/qVdBuXTfm6jLlnsBErDLlT40xr4vIUABjzFQR\nGQY8AqRjJZInjTFrHMe+CtzpeG8z8JAxJkVEbgPeAwKwus8ijTHdRORFYDSwN0sI/zLGxF0pPqeW\nKbuDi0UEjvGco5vgaCSkOBqKlxQROFo6WYoIFuw4zsNfbmTozTV5rkeIjR9Eqb8dT0xm/MIoftgU\nS6kivozoHMyAVtV1AN9GLrkPRkS8gGJZu7Y8Wb5LMNm5pIjAkXiOb4MMR/1DkTKX3J/zemQRPo5M\n4uuHWhJes5y9sasC7WxKOh8t38e0lTFkZsKgNjV4tGMtShbRAXy7OS3BiMg3wFAgA6vbqwQwyRgz\nzhmB2qlAJJjspKdC3M6/WzlHNkP8rotFBCckgB1Sk9bt/kWRGs3dbiYClb9lZBq+jTjMuwv3cDIp\nhd6NKzOqWx2qltEBfHfhzAQTaYwJFZEBQFPgOWCjMaaRc0K1T4FNMNlJPQvHtsCRTfwVvZbT+9ZR\nTS70LgqUC/57FoIqTaFCA/DVKf+Vcy2LiuPNubuJOnGGsOqleaFXXZpUK213WOoyzryT31dEfIFb\ngfeNMWki4p7VV+r6FfK/WERQKnwY3yyLZtr8CN67GdoVPXSVIoIs09/k4UwEKn/Zdew0b8zdxcq9\nJ6letigfDmhK9wYVdQDfw+UmwXwEHAC2ACtEpDqQL8Zg1JU93L4my6PiGfpHInNH9Kb6zf6XFhFc\nuDF02/cQ8al1kK8/VGrsSDqOyrXSNVw2E4HyfCdOJzNh4R6+23iY4n6+vHRLPQa2qq5VjPnEdVWR\niYiPMSbdBfHkKe0iu7qjf52n+8QV1CxfjO8ebo1PdlU7mZlwat+l9+cc2/rPIoIL9+c4cSYC5bnO\npaYzbUUMHy2PIT0zk/ta12BYp1qUKlrI7tBULjhzDGYE8BlwBvgYaIJ1r8m1TN3iljTB5OzXLUd5\nfMZmRnQOZmTX2rk76EIRwcXpby4tIqBE4KWtnMpNtIiggMjINPywMZbxC6OIO5NCz4YVebZ7CNXL\n+tsdmroGzhyDecAYM0lEugGlgYHAl4DHJxiVs96NK7M0Ko73luylfe1yNKteJueDfAo57rcJhbAH\nrG1ZiggulkxfMhNB8KWtHC0iyHdW7o3n9Tm72H38DE2qleLDe5rm7v+T8li5STAXOtB7Al8aY3aI\njrwVKK/+uz4bDpxixMxI5o1oR/HrmUgwSxHBRedO/V0mfWQjxCyFrTOt97IWEVxIPFpE4JH2nDjD\nG3N3sSwqnqplivD+3U3o1bCSDuAXALnpIvsMa2bkIKAx1l35y4wxzVwfnmtpF1nubTz4J3d89Ad9\nGldmwp2hrrmIMXD6aJb7czZZU+FknYlAiwg8RvyZFCb8vodZGw7hX9iH4Z2CuTe8OoV99EuCp3Nm\nF9mDQCgQY4w5JyJlgUE3GqDyLM2ql2ZYx1pMWryXDiHl+Xfjys6/iAiUrGI96v3b2naxiCDL9Dfr\np1+hiKApVGsNRXTiQzudT83g45UxTF2+j5T0TO4Lr8HwTsGU9tcB/IImV1VkIvJvoL3j5XJjzK8u\njSqPaAvm2qRnZNLvoz+Ijkti/hPtqVLKprXNM9LgxI4s099s+ruIoHBJaPsEtBwKhfTO77yUmWn4\ncfMRxi+I4vjpZLrXr8izPUIIKqcD+PmNM6vI3gKaA187Nt0FbDDGPH/DUdpME8y1O5Rwjh6TVlC/\nSklmDG6Ft5ebdE+lnrW609a8D3vmQfFK0GE0hA4Ab7uXPcr/1kSf5LU5u9h57DSNA0vyQq96tAjS\nAfz8ypkJZisQaoxVYyoi3sBmnSqm4PphYyxPfbeFZ7rV4bGOtewO558OroHfX4HY9dY6OZ1fgZBe\nOlbjAtFxZ3hz7m4W746jSqkijOpeh96NKuPlLl88lEs4cwwGoBRwyvG85HVHpfKF25tWYUlUHP/9\nfQ/tgsu532JP1cPhwYWwew4sfhVmDYDAFtB1LFRvbXd0+cLJpBQmLtrDjPWHKerrzXM9Qrg/vAZ+\nvjqAr/6WmxbMXcBbwFKskuX2WDdaznJ9eK6lLZjrl3guje6TVuDn682c4W0pWshNu6Ey0iHya1j2\nJpw5BrV7QJdXoHxduyPzSMlpGXy6ej8fLN3H+bQMBrSsxojOwZQtlrvVU1X+4NT1YESkEtY4DMB6\nwMsYc/TGQrSfJpgb88e+BO7+eC39m1flzdvdvMc09Rys+xBWTYTUJGh8N3QcDSUD7Y7MI2RmGmZv\nOcq4BVEc+es8XepWYHTPEGoG6PLaBZFLFhzLcvJDxphq1xWZG9EEc+Penr+bD5ftY+o9zejeoKLd\n4eTs3ClY+S6snwbiBS0fhrYjoYhOCX8l62ISeH3uLrbGJtKgSgle6FmP1jXL2h2WspGrE8xhY0zV\n64rMjWiCuXGp6Znc/uFqjvx5nvlPtKdCCQ+Z3uWvQ7D0Ddgy05oHrd1T0GII+NpUeu2GYuKTeGve\nbhbuPEGlkn48060Ot4ZW0QF8pS2Y3NAE4xz74pPoNXklzWuU4YtBLTzrF9Dx7VYhwN6FUKIKdHwe\nGt9VoKekOXU2lUmL9vD1ukMU9vHi0Y61eLBtkA7gq4tuOMGIyHtAdm8KcJ8xxuOnv9UE4zzfrDvE\n8z9t48VedXmo3U12h3Pt9q+ERa9YswUE1LUKAWp3L1ClzclpGXyx5gDvL43mbEo6d7WoxhNdahNQ\nXAfw1aWcUaZ8td+8+ltZXeKuFlVZGhXHO/OjCK9ZjnqVPez7R1A7eGixNcPz4rEwo7817UyXV6Fa\nS7ujcyljDL9uPcY783cT++d5OoWUZ3SPEIIrFLc7NOXhrquLLL/QFoxznTqbSreJKyhd1JfZw9p6\nbpdKRhps/hKWvQVJJyDkFuj8sjWbcz6z4cApXpuziy2H/6JupRK82KsubWqVszss5eZy24LRdUmV\n05TxL8T4fo3Zc8IaHPZY3r7WOjbDN0OnFyFmOXzQCmY/bs32nA8cOHmWoV9upN/UPzieeJ5xfRvx\n2+NtNbkop3LTu+OUp7q5dgAPtAni09X7ubl2AB1DPHh55EL+0P4ZaDbIUdo8HbZ+C60egTZPeOSs\nzX+dS2Xy4mi+XHsAX28vnuxam4faBbnvjbLKo2kXmXaROV1yWga3TlnNyaQU5j/RnnL55S7vPw/A\nktdh27fWfTPtnobmD3nEypsp6Rl8+cdBJi/eS1JKOnc2r8rIrrUpX9z9Y1fux5mTXU7OZnMiEGGM\n+eU643MLmmBcJ+r4GXq/v4p2tcrx8X1h+Wv1wmNbYNGrsG8xlKwKHV+ARne4ZWmzMYa5247z9vzd\nHDp1jptrB/B8z7rUqagD+Or6OXMMxg9rwbG9jkcjIBB4UEQm3lCUKt+qU7E4o3uEsHh3HF+tO2R3\nOM5VqTEM/BHu/QWKloWfh8LUdrBnobUqp5vYePBP/vPhGh77ZhNFC3nzvwda8MUDLTS5qDyTmxbM\nWqCNMSbD8doHWAm0BbYZY+q5PEoX0RaMaxljuP+zDayNSWDO8LbUKp8Pf7FlZsLOn63S5j/3Q/W2\n0PVVCMzxy53LHEo4x9sLdjNn6zECihfm6X/Vpm+zqu6zdo/yeM5swZQGss5o5w+UcSSclOuMTxUA\nIsK4fo2s9dhnRJKSnmF3SM7n5QUNbofH1kPP8XAyCj7uDLMGwsm9eRpK4rk0Xp+zky4TlrNkVxwj\nOgez7OkO3Nm8miYXZYvclI68A0SKyDL+nq7/DRHxBxa5MDaVD5Qv7sc7/2nEQ/+LYMLCPYzumU+n\nyfcpBC0GW9PM/PE+rHnPWo+m6b3Q4Tko7rqJQFPTM/lq7UEmL9lL4vk0+jUL5MmudahYUgfwlb2u\nZbr+Fo6XG/LDVP2gXWR56YWftvH1ukN8/VDLgnGvRVIcrBgHEZ+CdyFo/RiED7cm1nQSYwwLdhzn\nrXm7OZBwjra1yvF8z7qeN4uC8jjOXg/m31gtF4DlxphfbzA+t6AJJu+cT82g13srOZeSwbwR7Sjt\nX8jukPLGqRhY8hps/wGKlLHuq2n+IPjcWOl25OG/eH3OTjYc+JPg8sV4vlddOtQOyF/VesptOW0M\nRkTeAkYAOx2P4SLyRi6D6C4iUSISLSLPZfN+BxFJFJFIx+PlLO+NFJEdIrJdRGaIiJ9jez/H9kwR\nCcuyf1mGcMuBAAAgAElEQVQRWSoiSSLyfm7iU3mnSCFvJvdvQsLZFJ7/aRsF5v6rMjdB309hyDKo\n2BAWjIb3w6wbNjMzr/l0h0+dY/iMzdw6ZTX7T57ljdsaMm9EOzrWKa/JRbmd3FSRbQVCjTGZjtfe\nwGZjzFWXMHTstwfoCsQCG4C7jDE7s+zTAXjaGHPLZcdWAVYB9Ywx50XkW2CuMeZzEakLZAIfOY6N\ncBzjDzQBGgANjDHDcvrw2oLJe1OX7+Otebt5p28j7gjz+CWFrt2+JfD7K3B8K1RoCF3HQM3OOc7a\nfDo5jSlLo/ls9QEEGNzuJoZ2qEmxwnoHvsp7zphNOatSwCnH85K5PKYFEG2MiXEENBPog9UKyg0f\noIiIpAFFgaMAxphdjvNdsrMx5iywSkRq5fL8ygZD2t3E8qh4xszeQYsaZahRzt/ukPJWzU4Q1AF2\n/GiVNn/1Hwhqb83aXKXpP3ZPy8jkm3WHmLR4L3+eS+W2JlV4plsdKpXUhdGU+8tNmfKbwGYR+VxE\nvgA2Aq/n4rgqwOEsr2Md2y4XLiJbRWSeiNQHMMYcAcYDh4BjQKIxZmEurpkjERkiIhEiEhEfH++M\nU6pr4OUlvHtHY3y9vRgxK5K0jGvvJvJ4Xl7QsC8Mi4Ae78CJHTC9I3x3PyTsA6wB/IU7jtPtvyt4\nZfYO6lQozq/D2jLhjlBNLspj5JhgjDEzgFbAj8APQGusGy2dYRNQzdHd9h7wM4CIlMZq7QQBlQF/\nEbnHGRc0xkwzxoQZY8ICAgKccUp1jSqXKsIbtzVky+G/mLw4b+8VcSs+haDlwzA8Em5+1poJYEoL\nEmY9zsMfzmPIlxsRgU/uC+ObwS1pUCW3nQdKuYdcTddvjDlmjJnteBwH1ubisCNA1k72QMe2rOc9\nbYxJcjyfC/iKSDmgC7DfGBNvjEnDSm7huYlVeYZejSrRt1kgU5ZGs+HAqZwPyM/8SkDH5zl+/x+s\nKnkLJXZ+zcS4QfxSfznzH2lC57oVdABfeaTrXQ8mN//bNwDBIhIkIoWA/sDsS04iUlEcPzki0sIR\nTwJW11grESnqeL8zsOs6Y1Vuasy/6xNYuihPzIzkdHKa3eHY5kxyGu/M383NH+7kgfj+fB46E9+Q\nbjTe9xG+7zeFddMgPdXuMJW6ZtebYHKsMTXGpAPDgAVYyeFbY8wOERkqIkMdu/UFtovIFmAy0N9Y\n1gHfY3WhbXPEOQ1ARG4TkVisrro5IrLgwjVF5AAwAbhfRGJFxGPnSSsIihX2YWL/UI6fTubln7fb\nHU6eS8/I5Mu1B+kwbhkfLNtHjwYVWfp0Bwbf9i98+/8PHloC5evCvGdgSnPY9v11lTYrZZcrlimL\nyHtkn0gEuM8Y4/G3C2uZsnuYvHgvE37fw6T+ofQJza4OJH8xxrA0Ko435u4mOi6JFkFleLFXXRoF\nZrOAmTEQvRgWvQIntlszOXd5FWp2zPvAlXJwRpny1X7z6m9l5TSPdqjJ8j3xvPjTdppWK03VMkXt\nDslldhxN5PU5u1izL4GbyvkzbWAzuta7yhiLCAR3scqbt31nzQrw5a1wU0foMgYqh+Zl+Epdk2ta\n0VJEKjoG+fMFbcG4j8OnztFj0krqVirOzCGt893sv8cTkxm3IIofN8dSqogvIzoHM6BVdXy9r7GX\nOj0FNnxizXN2/hQ06AudXoQyQa4JXKlsOHUusiwn3WSM+efdYB5KE4x7+WlzLCNnbeHpf9VmWKdg\nu8NxiqSUdKYt38e0lTFkZsKgNjV4tGMtShbxvbETJyfC6snwxxTITIewB6x5zopp6b1yPWffyX/x\nvNcZj1I5ujW0Ckt3x/PfRXtpGxxAaNVsxiQ8RHpGJt9tjOXdhXs4mZRC78aVGdWtjvO6//xKQueX\noPlDsPxt2PAxRH5tzdjc+jEoXCzncyjlYtfagnnUGPOBC+PJU9qCcT+J59PoOWklvt7CnOHt8PfA\nubaWRcXxxtxd7DmRRFj10rzQqy5NqpV27UXj98CSsbDrV/AvDzePgmb3g/cNtpSUyoZLusjyG00w\n7mldTAL9p6/ljmZVebvvVedUdSu7jp3mjbm7WLn3JNXLFuW57iF0b1Axb2+SPLzBqjg7uNqaybnz\ny1Dv1hwn01TqWjhzyWSl8lTLm8ryaIeazIo4zLxtx+wOJ0cnTicz6vst9Jy8kq2xibx0Sz1+H3kz\nPRpWyvs78Ks2h/vnwN3fgo+fNb/Z9E6wf0XexqEU2oLRFoybSsvI5D8fruFgwjkWPNHeLZf/PZea\nzrQVMXy0PIb0zEzua12DYZ1qUaqomyymlpkBW2fBktfhdCzU6mKVNldsaHdkysNpF1kuaIJxbzHx\nSfSavIqm1Uvx5QMt8XKT0uWMTMMPG2MZvzCKuDMp9GpYiVHd61C9rJsuPZCWDBumw4rxVvVZozug\n4wtQurrdkSkPpV1kyuPdFFCMV3rXY3V0Ap+s2m93OACs3BtPr8krGfXDVqqULsIPj7RmyoCm7ptc\nAHz9IPxxGLEF2j4BO3+xVtWcPxrOJtgdncrHtAWjLRi3Zoxh6FcbWbI7jp8fa0P9yvZMWb/nxBle\nn7OL5XviqVqmCM92D6GXHWMszpB4BJa/BZu/gkLFoM1waPUoFHLjJKncinaR5YImGM/w59lUuk9a\nQXE/X34d1pYihbzz7NpxZ5L57+97mbXhEMUK+/B4p2DuDa9OYZ+8i8Fl4qNg0asQNQeKVYAOz0GT\ne8Hb80rDVd7SLjKVb5T2L8T4fo2JjkvizXl5s2rD+dQM3lu8l47jlvFdxGHuC6/B8mc6Mrj9Tfkj\nuQAE1IG7voEHFkDpIPhtJHzQEnbOtibZVOoG6VcV5RHaBQfwUNsgPl61n5trB9C5bgWXXCcz0/Dj\n5iOMXxDF8dPJdK9fkWd7hBBULh93H1VrBQ/Mh6h5sPhV+HYgVAmDrmOhRhu7o1MeTLvItIvMY6Sk\nZ9Dn/dXEn0lh/hPtCShe2KnnXxN9ktfm7GLnsdM0DizJC73q0SKojFOv4fYy0mHLDFj6Bpw5CsHd\noMsrUKG+3ZEpN6JjMLmgCcbz7Dlxht7vrSK8Zlk+vb+5UwbZo+PO8Obc3SzeHUeVUkUY1b0OvRtV\ndpuyaFuknYd1H8GqCZB8GhrfBR2fh1JVcz5W5XuaYHJBE4xn+mLNAV6ZvYOxfepzb+sa132ek0kp\nTFy0hxnrD1PU15vHOtXi/vAa+PnmkzEWZzh3Clb910o2AC0GQ7unoGgBa9mpS2iCyQVNMJ7JGMMD\nn29gzb4Efn28LbUrFL+m45PTMvhk1X4+XLaP82kZDGhZjRGdgylbzLldbvlKYiwsfdOasblwCet+\nmpZDoVD+XRxOXZkmmFzQBOO54s+k0H3iCgKKF+aXYW1yVdmVmWn4ZcsRxs2P4mhiMl3qVmB0zxBq\nBujU9rl2YicsHgt75kHxStBhNIQO0NLmAkbLlFW+FlC8MOP6NWL38TOMmx+V4/5rYxLoM2U1I2dt\noUyxQswY3IqP7wvT5HKtKtSDu2fCoHlQsir8Ohw+DIfdc7S0Wf2DJhjlsTqFVGBgq+p8vGo/q/ae\nzHafffFJDP5fBP2nreVkUgr/vbMxsx9rS+uaZfM42nymejg8uBDu/ApMJsy8Gz7tBgf/sDsy5Ua0\ni0y7yDza+dQMer+/itPn01jwRHtK+1szGZ86m8qkRXv4et0hCvt48WjHWjzYNkgH8F0hIx0iv7LG\naJKOQ52e1jo05evaHZlyER2DyQVNMPnDjqOJ3DplNZ1CyjOpfxM+X3OAKUuiOZeWQf/mVXmiS22n\n3zOjspF6DtZ9CKsmQmoShN4NHZ6HklXsjkw5mSaYXNAEk39MXxHD63N3Uca/EKfOptIppDyje4QQ\nfI0VZsoJzp2Cle/C+mkgXtDyYWg7Eoq4eNlolWc0weSCJpj8IzPT8PBXGzlxOplnu4fQplY5u0NS\nfx2yZgTYMhP8Slj3z7QYAr5F7I5M3SBNMLmgCUapPHB8OywaA9G/Q4kq1owAje8CLx0P81RapqyU\ncg8VG8A938N9v1rLAvzyGHzYxppcswB/wS0INMEopfJGUHsYvAT6fQEZqTCjP3zWAw6tszsy5SKa\nYJRSeUcE6t8Kj62DXhMgYR98+i+YOQDi99gdnXIyTTBKqbzn7QvNH4Thm6HjixCz3FrsbPZwOH3U\n7uiUk2iCUUrZp3AxuPkZGBEJLR6GyG9gclNrKefzf9kdnbpBLk0wItJdRKJEJFpEnsvm/Q4ikigi\nkY7Hy1neGykiO0Rku4jMEBE/x/Z+ju2ZIhJ22flGO64VJSLdXPnZlFJO5F8OerwFj0dA3d7WOjST\nQ2HN+5CWbHd06jq5LMGIiDcwBegB1APuEpF62ey60hgT6niMdRxbBRgOhBljGgDeQH/H/tuB24EV\nl12vnmOf+kB34ANHDEopT1G6BvxnOjy8Aio3hYUvwPthEDkDMjPsjk5dI1e2YFoA0caYGGNMKjAT\n6HMNx/sARUTEBygKHAUwxuwyxmQ3fW4fYKYxJsUYsx+IdsSglPI0lRrDwB/h3l+gaFn4eShMbQd7\nFmppswdxZYKpAhzO8jrWse1y4SKyVUTmiUh9AGPMEWA8cAg4BiQaYxY66XpKKU9xUwcYvBT6fgpp\n5+CbfvBFb4jdaHdkKhfsHuTfBFQzxjQC3gN+BhCR0lgtkiCgMuAvIvc444IiMkREIkQkIj4+3hmn\nVEq5kpcXNPgPPLYeeo6H+N3wcSf49l44GW13dOoqXJlgjgBVs7wOdGy7yBhz2hiT5Hg+F/AVkXJA\nF2C/MSbeGJMG/AiE3+j1HNeZZowJM8aEBQQEXOtnUkrZxacQtBhslTZ3GA3Ri2FKC/htJJw5bnd0\nKhuuTDAbgGARCRKRQlgD8LOz7iAiFUVEHM9bOOJJwOoaayUiRR3vdwZ25XC92UB/ESksIkFAMLDe\nqZ9IKWW/wsWhw3NWomn+IGz6H0xuAkteg+TTdkensnBZgjHGpAPDgAVYyeFbY8wOERkqIkMdu/UF\ntovIFmAy0N9Y1gHfY3WhbXPEOQ1ARG4TkVigNTBHRBY4rrcD+BbYCcwHHjPGaNmJUvlVsfLQc5zV\ndVanB6wYZ5U2r/0Q0lPsjk6hsynrbMpK5RdHN8Pvr8D+5VCqGnR6CRr0tcZwlFPpbMpKqYKlchO4\nbzYM/An8SsGPg2Fae4hepKXNNtEEo5TKX2p2giHL4faPrTGZr/4D//s3HNlkd2QFjiYYpVT+4+UF\njfrBsAjo8Q6c2AHTO8J391szOKs8oQlGKZV/+RSClg/D8EhoPwr2LLBKm+c8DUlxdkeX72mCUUrl\nf34loNMLVqJpeh9EfAqTQmHpm5Byxu7o8i1NMEqpgqN4BbhlglXaHNwVlr9lJZp10yA91e7o8h1N\nMEqpgqdcLbjjC3hoCZSvC/OegSnNYdv3kJlpd3T5hiYYpVTBFdgM7vsVBvwAhYrBDw/C9A6wb6nd\nkeULmmCUUgWbCAR3gYdXwm0fwblT8OWt8OVtcGyL3dF5NB+7A3A3aWlpxMbGkpysq+i5Iz8/PwID\nA/H19bU7FJXfeHlB4/5Q71aI+MSaeuaj9tCwH3R8AcoE2R2hx9GpYi6bKmb//v0UL16csmXL4piH\nU7kJYwwJCQmcOXOGoCD9YVculpwIqyfBHx9AZro1sWb7Z6zlnQs4nSrmOiUnJ2tycVMiQtmyZbV1\nqfKGX0no/LI1a3OTAbB+ulVxtvwdSEmyOzqPoAkmG5pc3Jf+26g8V6IS9J4Ej66Fmh1g6evW8gAb\nPoaMNLujc2uaYJRSKjcCasOdX8GDi6BsLZjzlDUrwI6fdDLNK9AEU8AtW7aMW2655Yb3uZrdu3fT\nunVrChcuzPjx46+43/33309QUBChoaGEhoYSGRl53ddUymWqNodBc+Hub8HHz5rfbHon2L/C7sjc\njlaRuTljDMYYvDx4TYsyZcowefJkfv755xz3HTduHH379s2DqJS6ASJQuxvU6gJbZ8GS1+GL3tbr\nLmOgYkO7I3QLnvtbKx87cOAAderU4d5776VBgwZ8+eWXtG7dmqZNm9KvXz+SkqwBxrlz5xISEkKz\nZs0YPnz4VVsZ69evp3Xr1jRp0oTw8HCioqL+sc+YMWMYOHAgrVu3Jjg4mOnTp198Lykpib59+xIS\nEsKAAQO4UH04duxYmjdvToMGDRgyZAjZVSWWL1+e5s2ba2mxyn+8vCH0bnh8I/zrNYiNgKnt4Mch\n8OdBu6OznbZgruLVX3ew86hz1/iuV7kEr/Sun+N+e/fu5YsvvqBWrVrcfvvtLFq0CH9/f95++20m\nTJjAqFGjePjhh1mxYgVBQUHcddddVz1fSEgIK1euxMfHh0WLFvH888/zww8//GO/rVu3snbtWs6e\nPUuTJk3o1asXAJs3b2bHjh1UrlyZNm3asHr1atq2bcuwYcN4+eWXARg4cCC//fYbvXv3ZurUqQAM\nHTr0H9e4mtGjRzN27Fg6d+7MW2+9ReHCha/peKVs4esH4Y9Dk4Gw6r+wbqo1NtN8MLR7CvzL2h2h\nLbQF46aqV69Oq1atWLt2LTt37qRNmzaEhobyxRdfcPDgQXbv3s1NN9108X6QnBJMYmIi/fr1o0GD\nBowcOZIdO3Zku1+fPn0oUqQI5cqVo2PHjqxfvx6AFi1aEBgYiJeXF6GhoRw4cACApUuX0rJlSxo2\nbMiSJUsunnfo0KHXnFzefPNN9uzZw4YNGzh16hRvv/32NR2vlO2KlIKur8Ljm6DRnbDuQ5gcCivG\nQ+pZu6PLc9qCuYrctDRcxd/fH7DGYLp27cqMGTMuef9aB8BfeuklOnbsyE8//cSBAwfo0KFDtvtd\nXgZ84XXWloS3tzfp6ekkJyfz6KOPEhERQdWqVRkzZswN3aNSqVKli9caNGjQVQsClHJrJatAn/eh\n9TBYPBaW/J91H02H56xWjnfB+NWrLRg316pVK1avXk10dDQAZ8+eZc+ePdSpU4eYmJiLLYlZs2Zd\n9TyJiYlUqVIFgM8///yK+/3yyy8kJyeTkJDAsmXLaN68+RX3vZBMypUrR1JSEt9///01fLJ/Onbs\nGGAl1Z9//pkGDRrc0PmUsl35ELjrG3hgAZSuDr89AR+0gp2zC0RpsyYYNxcQEMDnn3/OXXfdRaNG\njWjdujW7d++mSJEifPDBB3Tv3p1mzZpRvHhxSpYsecXzjBo1itGjR9OkSRPS09OvuF+jRo3o2LEj\nrVq14qWXXqJy5cpX3LdUqVIMHjyYBg0a0K1bt0uS0dSpUy+Owxw/fpzAwEAmTJjAa6+9RmBgIKdP\nW2NbPXv25OjRowAMGDCAhg0b0rBhQ06ePMmLL754TX9XSrmtaq2sJNN/BogXfDsQPu4CB1bbHZlL\n6Vxkl81FtmvXLurWrWtTRNcmKSmJYsWKYYzhscceIzg4mJEjR173+caMGUOxYsV4+umnnRil83nS\nv5FS/5CRDltmwNI34MxRCO4GXV6BCvZ1yV8rnYusAJg+fTqhoaHUr1+fxMREHn74YbtDUkrlxNsH\nmg60Spu7jIFDa+HDNvDTI/DXYbujcyptwXhwCyY7n332GZMmTbpkW5s2bZgyZYpNETmfp/8bKXWJ\nc6dg1QRr2WaAlkOg7ZNQtIy9cV1FblswmmDyWYIpCPTfSOVLfx2GZW9C5DdQuAS0Gwkth4JvEbsj\n+wftIlNKKU9Sqirc+gE8shqqt4ZFY2ByU9j0P2vcxgNpglFKKXdSoT7cPQvun2vdTzP7cfgwHHbP\n8bjSZk0wSinljmq0gQd/t5YIMJkw8274tJtVFOAhNMEopZS7EoG6va3FznpPsibQ/LQbzLgL4nbb\nHV2ONMEUcHmxHszXX39No0aNaNiwIeHh4WzZsuW6z6VUgeTtA83ut5Zv7vwyHFgFH7aGXx6DxCN2\nR3dFLk0wItJdRKJEJFpEnsvm/Q4ikigikY7Hy1neGykiO0Rku4jMEBE/x/YyIvK7iOx1/Fnasb2Q\niHwmIttEZIuIdHDlZ8srxhgyMzPtDuOGBAUFsXz5crZt28ZLL73EkCFD7A5JKc9UqKg1O/OILdDy\nEdj6LbzXFH5/Bc7/aXd0/+CyGddExBuYAnQFYoENIjLbGLPzsl1XGmNuuezYKsBwoJ4x5ryIfAv0\nBz4HngMWG2PeciSt54BngcEAxpiGIlIemCcizY0x1//bed5zcHzbdR+erYoNocdbV93lwIEDdOvW\njZYtW7Jx40ZGjRrF1KlTSUlJoWbNmnz22WcUK1aMuXPn8uSTT+Lv70+bNm2IiYnht99+y/ac69ev\nZ8SIESQnJ1OkSBE+++wz6tSpc8k+Y8aMYd++fURHR3Py5ElGjRrF4MGDgb/Xg9m+fTvNmjXjq6++\nQkQYO3Ysv/76K+fPnyc8PJyPPvroHxNmhoeHX3zeqlUrYmNjr+dvTil1QdEy0P0NaPmwNSPA6kmw\n8XMr+bQYYi0f4AZc2YJpAUQbY2KMManATKDPNRzvAxQRER+gKHDUsb0P8IXj+RfArY7n9YAlAMaY\nOOAvIMc6bXe1d+9eHn30UZYvX84nn3zCokWL2LRpE2FhYUyYMIHk5GQefvhh5s2bx8aNG4mPj7/q\n+S6sB7N582bGjh3L888/n+1+W7duZcmSJfzxxx+MHTv24jxhmzdvZuLEiezcuZOYmBhWr7bmUBo2\nbBgbNmxg+/btnD9//mKCyzoXWVaffPIJPXr0uJG/GqXUBaWrw+0fwdCVENgcfn8J3msGm7+GzAy7\no3PpdP1VgKzzHsQCLbPZL1xEtgJHgKeNMTuMMUdEZDxwCDgPLDTGLHTsX8EYc8zx/DhQwfF8C/Bv\nEZkBVAWaOf5cf92fIIeWhitdWA/mt99+u7geDEBqaurFCS8vXw9m2rRpVzxfYmIi9913H3v37kVE\nSEtLy3a/C+vBFClS5OJ6MKVKlbq4HgxwcT2Ytm3bsnTpUt555x3OnTvHqVOnqF+/Pr179852LZil\nS5fyySefsGrVqhv961FKZVWxIdzzPexfYXWX/fIorHnPmoqmdjerWMAGdg/ybwKqGWMaAe8BPwM4\nxlX6AEFAZcBfRO65/GBjTUNwoTD8U6wkFgFMBNYA/0jhIjJERCJEJCKnb/12unw9mMjISCIjI9m5\ncyeffPLJNZ/vwnow27dv59dff73iui3Xsx7M999/z7Zt2xg8ePAVz7t161YeeughfvnlF8qWLZir\n+ynlckHtYfAS6PcFZKTCjDvhs55w+Pq/Z98IVyaYI1gtiAsCHdsuMsacNsYkOZ7PBXxFpBzQBdhv\njIk3xqQBPwIXOvJPiEglAMefcY7j040xI40xocaYPkApYM/lQRljphljwowxYQEBAc78vC6RH9aD\nOXToELfffjtffvkltWvXvmqcSqkbJAL1b4XH1kGvCZAQDZ90hZkDIP4fvxJdypUJZgMQLCJBIlII\na5B+dtYdRKSiOL4ii0gLRzwJWF1jrUSkqOP9zsAux2Gzgfscz+8DfnEcX1RE/B3PuwLp2RQUeJz8\nsB7M2LFjSUhI4NFHHyU0NJSwMI8dGlPKc3j7QvMHrdLmji9CzHJrsbPZw+H0sZyPdwKXTnYpIj2x\nuqu8gU+NMa+LyFAAY8xUERkGPAKkY421PGmMWeM49lXgTsd7m4GHjDEpIlIW+BaoBhwE7jDGnBKR\nGsACIBOrpfSgMebg1eLz9MkudT0YpVSunT0JK8bDho/BywfaPwXtn7muU+V2skuXLgzt6Paae9m2\nqVmevw+8f4VjXwFeyWZ7AlaL5vLtB4A6l2/Pz6ZPn84XX3xBamoqTZo00fVglFJX5l/OKly6UNqc\nB/Oa6XT9HtyCyY6uB6OUyhVjrru6zC1aMCrvDRo0iEGDBtkdhlLK3eVB6bLdZcpuqSC36tyd/tso\n5Tk0wVzGz8+PhIQE/UXmhowxJCQk4OfnHtNgKKWuTrvILhMYGEhsbGyOU68oe/j5+V2cUUAp5d40\nwVzG19f34vQrSimlrp92kSmllHIJTTBKKaVcQhOMUkoplyjQN1qKSDzWdDPuohxw0u4grsLd4wON\n0RncPT5w/xjdPT64sRirG2NynC24QCcYdyMiEbm5O9Yu7h4faIzO4O7xgfvH6O7xQd7EqF1kSiml\nXEITjFJKKZfQBONerrzmsXtw9/hAY3QGd48P3D9Gd48P8iBGHYNRSinlEtqCUUop5RKaYPKYiHQX\nkSgRiRaR566wTwcRiRSRHSKy3N1iFJGSIvKriGxxxJin6wOIyKciEici26/wvojIZEf8W0WkaV7G\nl8sYBzhi2yYia0SksTvFl2W/5iKSLiJ98yq2LNfOMUY7f1Zy8W9s68+JI4aqIrJURHY6YhiRzT6u\n+3kxxugjjx5YS0fvA24CCgFbgHqX7VMK2AlUc7wu74YxPg+87XgeAJwCCuVhjO2BpsD2K7zfE5gH\nCNAKWGfDv3VOMYYDpR3Pe+R1jDnFl+X/whKsVWn7uuHfod0/KznFZ+vPieO6lYCmjufFgT3Z/Dy7\n7OdFWzB5qwUQbYyJMcakAjOBPpftczfwozHmEIAxJs4NYzRAcRERoBjWD056XgVojFnhuOaV9AH+\nZyxrgVIiUilvorPkFKMxZo0x5k/Hy7VAnk4RnYu/Q4DHgR+AvP4/COQqRlt/VnIRn60/JwDGmGPG\nmE2O5//f3v2FSlVFcRz//kDFTJAyCLLkWqkVIZn2D42ygigEE4TqQUnCCAuUHhJ6MEEoiKgIsyyL\n/iAGiaRFKD4opWiZKf5BS9HKmz1koKlJYK4e9ram0Tsz13vPOSP8Pi+XObM5s869rLvO2efM2seA\n3cCQumGF5YsLTLmGAAdrXndy9h97BHCJpHWStkiaVlp0SSsxLgCuBw4BO4BZEXG6nPBa0soxtJPH\nSWeQbUPSEGAy8GbVsTRQda4001Z5IqkDGA18XfdWYfnidv3tpw8wBrgXuAjYKGlTRPxQbVj/cz+w\nDYDebXoAAAPwSURBVLgHuAZYI+mriPij2rAuPJImkArM+KpjqfMaMCciTquEpXXPU7vnStvkiaSB\npKvR2WV+vq9gyvULcFXN6yvztlqdwOqIOBERh4EvgTJvALcS43TS1ERExD7gAHBdSfG1opVjqJyk\nUcBiYFJE/F51PHXGAh9L+hGYAiyU9FC1IZ2l6lxppi3yRFJfUnFZEhHLzzGksHxxgSnXZmC4pGGS\n+gGPACvrxqwAxkvqI2kAcBtp3rSdYvyZdNaIpMuBkcD+EmNsZiUwLT8dcztwNCJ+rTqoWpKGAsuB\nqW10xv2viBgWER0R0QEsA2ZGxKcVh1Wv6lxppvI8yfd/3gV2R8QrXQwrLF88RVaiiDgl6WlgNekJ\nnfciYpekJ/P7b0XEbkmrgO3AaWBxRDR8lLTsGIH5wPuSdpCePJmTzyBLIWkpcDdwmaRO4Hmgb018\nX5CejNkH/Ek6kyxVCzHOBQaTrgwATkWJzRFbiK9yzWKsOlda+B1WmifZOGAqsEPStrztOWBoTZyF\n5Yu/yW9mZoXwFJmZmRXCBcbMzArhAmNmZoVwgTEzs0K4wJiZWSFcYMzMrBAuMGYXgNyW/vOejjEr\nkwuMWS/I34J2PpnVcEKYnSdJHUoLs30I7ASmStoo6TtJn+QGg0h6UNKe3PH39UZXGZJuzfvYqrQQ\n2chzjJkn6aM8bq+kGTVvD5S0LH/ektwqBElzJW2WtFPS22e2mxXJBcasZ4YDC4G7SF2R74uIm4Fv\ngWck9QcWAQ9ExBjSwlON7AHujIjRpHYyL3QxbhSpS+8dwFxJV+Tto4HZwA2kRePG5e0LIuKWiLiR\n1Hl4YreP1Kyb3IvMrGd+iohNkiaS/qlvyBcH/YCNpO65+yPiQB6/FHiiwf4GAR9IGk5asKpvF+NW\nRMRJ4KSktaSF4o4A30REJ0DuPdUBrAcmSHoWGABcCuwCPju/QzZrjQuMWc+cyD8FrImIR2vflHRT\nN/c3H1gbEZPzAlHruhhX30TwzOu/arb9DfTJV1ELgbERcVDSPKB/N+My6zZPkZn1jk3AOEnXAki6\nWNII4Hvg6lwsAB5usp9B/LcWx2MNxk2S1F/SYFJH380Nxp4pJofzfaEpTWIw6xUuMGa9ICJ+IxWE\npZK2k6fH8jTWTGCVpC3AMeBog129BLwoaSuNZxi2A2tJhW1+RBxqENsR4B3SgwiraVyMzHqN2/Wb\nFUzSwIg4np/cegPYGxGv9mB/84DjEfFyb8VoVgRfwZgVb0a+4b6LNAW2qOJ4zErhKxizCkiaDsyq\n27whIp6qIh6zIrjAmJlZITxFZmZmhXCBMTOzQrjAmJlZIVxgzMysEC4wZmZWiH8AOOzQQ4SvjgEA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1041486a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### 降低学习率，调整树的数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read data\n",
    "dpath = './data/'\n",
    "test = pd.read_csv(dpath + 'RentListingInquries_FE_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_pred = gsearch5_1.predict_proba(test.as_matrix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.12301204,  0.39477015,  0.48221782],\n",
       "       [ 0.27817801,  0.4490819 ,  0.27274013],\n",
       "       [ 0.03222747,  0.09889407,  0.86887848],\n",
       "       ..., \n",
       "       [ 0.05551888,  0.33070067,  0.61378044],\n",
       "       [ 0.50652611,  0.38496986,  0.108504  ],\n",
       "       [ 0.04465988,  0.35919863,  0.59614152]], dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
